{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "cb1a3cd7",
   "metadata": {},
   "source": [
    "直接学习游戏环境"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4a033743",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAR8AAAEXCAYAAACUBEAgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAc2klEQVR4nO3dfXBU9d028Gs3+5LXs0mA7BJJILegkOGlEjCsjKUtKdGmVmr6jGUYm1oeHenC8NJhaqzg1OlMePAerbYQO+NU+EfTwbtoRVBzBwg6LG+BaAgY0RtNStgECdlNAtkku9/nD8m5WUDcJdn8duP1mTkz5JzvJtcS9mLPObtnDSIiICIaYUbVAYjou4nlQ0RKsHyISAmWDxEpwfIhIiVYPkSkBMuHiJRg+RCREiwfIlKC5UNESigrn82bN2PSpElITExEYWEhDh8+rCoKESmgpHz+8Y9/YO3atXjmmWdw7NgxzJo1C8XFxWhvb1cRh4gUMKh4Y2lhYSHmzp2Lv/71rwCAYDCInJwcrFy5Ek8++eS33j4YDKK1tRVpaWkwGAzRjktEYRIRdHV1ITs7G0bjzZ/bmEYok66vrw91dXUoLy/X1xmNRhQVFcHtdt/wNn6/H36/X//67NmzyM/Pj3pWIro1LS0tmDBhwk1nRrx8vvrqKwQCAdjt9pD1drsdn3zyyQ1vU1FRgT/+8Y/XrW9paYGmaVHJSUSR8/l8yMnJQVpa2rfOjnj53Iry8nKsXbtW/3rwDmqaxvIhikHhHA4Z8fIZO3YsEhIS0NbWFrK+ra0NDofjhrexWq2wWq0jEY+IRsiIn+2yWCwoKChATU2Nvi4YDKKmpgZOp3Ok4xCRIkp2u9auXYuysjLMmTMHd999N/785z+jp6cHjz76qIo4RKSAkvJ5+OGHcf78eWzYsAEejwff+9738O677153EJqIRi8lr/MZKp/PB5vNBq/XywPORDEkkscm39tFREqwfIhICZYPESnB8iEiJVg+RKQEy4eIlGD5EJESLB8iUoLlQ0RKsHyISAmWDxEpwfIhIiVYPkSkBMuHiJRg+RCREiwfIlKC5UNESrB8iEgJlg8RKcHyISIlWD5EpATLh4iUYPkQkRIsHyJSguVDREqwfIhICZYPESnB8iEiJVg+RKQEy4eIlGD5EJESLB8iUoLlQ0RKsHyISAmWDxEpEXH57N+/Hw888ACys7NhMBjw5ptvhmwXEWzYsAHjx49HUlISioqKcPr06ZCZjo4OLF26FJqmIT09HcuWLUN3d/eQ7ggRxZeIy6enpwezZs3C5s2bb7h906ZNeOmll/Dyyy/j0KFDSElJQXFxMXp7e/WZpUuXorGxEdXV1di5cyf279+Pxx9//NbvBRHFHxkCALJjxw7962AwKA6HQ5577jl9XWdnp1itVnn99ddFROTkyZMCQI4cOaLP7N69WwwGg5w9ezasn+v1egWAeL3eocQnomEWyWNzWI/5nDlzBh6PB0VFRfo6m82GwsJCuN1uAIDb7UZ6ejrmzJmjzxQVFcFoNOLQoUM3/L5+vx8+ny9kIaL4Nqzl4/F4AAB2uz1kvd1u17d5PB5kZWWFbDeZTMjMzNRnrlVRUQGbzaYvOTk5wxmbiBSIi7Nd5eXl8Hq9+tLS0qI6EhEN0bCWj8PhAAC0tbWFrG9ra9O3ORwOtLe3h2wfGBhAR0eHPnMtq9UKTdNCFiKKb8NaPnl5eXA4HKipqdHX+Xw+HDp0CE6nEwDgdDrR2dmJuro6fWbPnj0IBoMoLCwczjhEFMNMkd6gu7sbn332mf71mTNnUF9fj8zMTOTm5mL16tX405/+hClTpiAvLw/r169HdnY2Fi9eDACYNm0a7rvvPjz22GN4+eWX0d/fjxUrVuCXv/wlsrOzh+2OEVGMi/RU2t69ewXAdUtZWZmIfH26ff369WK328VqtcrChQulqakp5HtcuHBBlixZIqmpqaJpmjz66KPS1dUVdgaeaieKTZE8Ng0iIgq775b4fD7YbDZ4vV4e/yGKIZE8NuPibBcRjT4sHyJSguVDREqwfIhICZYPESkR8et8iGKFBAII9vcDIjAYjTCYzTAY+f9pvGD5UNwREfhbW/HV++/D9/HHCPh8MGdmQisowNiiIpjHjIHBYFAdk74Fy4fiioigu7ERX27eDH9rK3DlZWp958+j59NP4T16FBNXrEDSxIksoBjH56gUV/ytrWjesgX+s2f14tGJ4NKnn6K5shIDXq+agBQ2lg/FDQkG0f7OO+j9979vOtdz6hQ69u1DHL54/zuF5UNxo7elBRf37w9rtvPIkSinoaFi+VBcEBF0nzqFAV5Cd9Rg+VB8EEHHvn2qU9AwYvlQXOhtacHlCC6fa7Rao5iGhgPLh+LC5ZYWBLq6wp7P/P73o5iGhgPLh2KeiKDzwIGw582ZmUi5806+zifGsXwo5vV3dKDnmo/cvpmkiRNhvebjmSj2sHwo5l3+4gv0XfOJJzeTMX8+wPd4xTz+hiimiQg6Pvjg+lczfwNTejpSZ8zgLlccYPlQTBu4eBHdJ0+GPZ/8H/8By9ixUUxEw4XlQzHt8pdfov/8+fCGDQbY5syBwcT3S8cDlg/FLBHBRbcbEgiENZ+QkgLtrru4yxUnWD4UswKXLuHS55+HPW/JyoI5MzOKiWg4sXwoZvnPnUNvc3PY87Y5c2BMTIxiIhpOLB+KSSIC75EjCPr9Yc0brVZkOJ3c5YojLB+KSUG/H75jx8KeT8zNhcXhiGIiGm4sH4pJfo/nWy8adrW06dORkJwcxUQ03Fg+FHNEBF0ffYRAT09Y84aEBGTcey93ueIMy4dijgwM4OKHH4Y9n3T77bCOHx/FRBQNLB+KOb1nz6K3tTXs+bT8fJhSUqKYiKKB5UMxRURw6fTp8K/dk5CA9HvuiW4oigqWD8UWEXiPHg17PHnSJCRNnBjFQBQtLB+KKf0XLuDSZ5+FPZ82cyZfWBinWD4UM0QEPadPoy/MN5IaTCbYCgqinIqiheVDMaXjgw/CnrVmZyN5yhSeYo9TEZVPRUUF5s6di7S0NGRlZWHx4sVoamoKment7YXL5cKYMWOQmpqK0tJStLW1hcw0NzejpKQEycnJyMrKwrp16zAwMDD0e0Nxrb+jI6JdLr6XK75FVD61tbVwuVw4ePAgqqur0d/fj0WLFqHnqheDrVmzBm+//Ta2b9+O2tpatLa24qGHHtK3BwIBlJSUoK+vDwcOHMC2bduwdetWbNiwYfjuFcWly198gb5r/qP6JgaLBdqsWVFORNFkkCF8oPX58+eRlZWF2tpafP/734fX68W4cePw2muv4Re/+AUA4JNPPsG0adPgdrsxb9487N69Gz/96U/R2toKu90OAHj55Zfx+9//HufPn4fFYvnWn+vz+WCz2eD1eqFp2q3GpxgiIvji+efRUVsb1rz1ttsw9T//k6/viTGRPDaHdMzH6/UCADKvXEOlrq4O/f39KCoq0memTp2K3NxcuN1uAIDb7caMGTP04gGA4uJi+Hw+NDY23vDn+P1++Hy+kIVGl/6ODnSfOhX2vG3uXCQkJUUxEUXbLZdPMBjE6tWrMX/+fEyfPh0A4PF4YLFYkJ6eHjJrt9vh8Xj0mauLZ3D74LYbqaiogM1m05ecnJxbjU0xqq+9Hf0XLoQ3nJCA5Lw8gAea49otl4/L5cKJEydQVVU1nHluqLy8HF6vV19aIvjYXIp9IoLOgwfDvlyqKS0N2uzZPMsV527pStsrVqzAzp07sX//fkyYMEFf73A40NfXh87OzpBnP21tbXBcudaKw+HA4cOHQ77f4Nkwxzdcj8VqtcLKz94etQKXLqHrxImw520FBTzWMwpE9MxHRLBixQrs2LEDe/bsQV5eXsj2goICmM1m1NTU6OuamprQ3NwMp9MJAHA6nWhoaED7VR8CV11dDU3TkJ+fP5T7QnGqz+MJ/3KpCQlImzGDn1AxCkT0G3S5XHjttdfw1ltvIS0tTT9GY7PZkJSUBJvNhmXLlmHt2rXIzMyEpmlYuXIlnE4n5s2bBwBYtGgR8vPz8cgjj2DTpk3weDx4+umn4XK5+OzmO0hE0Hn4cNiXSzWlpCCNp9hHhYjKp7KyEgDwgx/8IGT9q6++il//+tcAgBdeeAFGoxGlpaXw+/0oLi7Gli1b9NmEhATs3LkTy5cvh9PpREpKCsrKyvDss88O7Z5QXAr6/RG9kVSbPRvma05oUHwa0ut8VOHrfEaPS2fO4NOnngrvqoVGIyatWYMxCxZEPxjdkhF7nQ/RUER6uVRTWhpSp02LcioaKSwfUicQiOgsV2p+Pj+HfRRh+ZAyfRG+kTRj/nwYjPwnO1rwN0lKiAi6GxvRf/FiWPOmjAykTJkS5VQ0klg+pIYIOt1uIMzzHSm33w7LNW/LofjG8iEl+js6cOl//ifs+Yx77+V7uUYZlg+NOBHBpc8+Q99Vr3K/GXNGBlKmTuV7uUYZlg8pcWHv3rBnk/LyYM3KimIaUoHlQyOu/8IFXPr887Dn0wsLAZ7lGnX4G6UR1/vvf4e9y5WQkoLkyZO5yzUKsXxoRA2+kTRcVoeDHwo4SrF8aEQFurvRffJk2PMZ994Lg9kcxUSkCi+KQiPKf+4cer/8EsCVZ0F9ffi8qws2iwW3p6XBeNXuVUJKCrRZs7jLNUqxfGhEXTxwABIIQETQ3NOD9cePo8nrRYrJhP97xx14OC8PCVfKxjp+PKy33aY4MUULd7toRAUuXQIACID/19CAk52dCIjA19+Pv546hRNXvd3CNncujLzA3KjF8iFlfP39IV/3BYPwX7mIvDExEenz5nGXaxRj+dCIGvysLQOAHzocMF1VLndoGiampupz5owMFRFphPCYD42oMT/8Ib6qrkaguxtlkycjzWzGf587h/FJSXjsjjuQdeWz1zMWLICJV6kc1Vg+NKISc3MxfskSnN22Daa+PvyfSZPwi0mTMPj8x2AwIDU/H/bFi3ntnlGO5UMjymA0Ytx99wEA2v7rv9B/8SIMVy6rYTCbod11FyYsWwbLlY/gptGL5UMjzmg2I6ukBLbZs+E7fhx+jwfGpCSkTpuG1GnT+Bns3xEsH1LCYDQi8bbbkMjX8XxncaeaiJRg+RCREiwfIlKC5UNESrB8iEgJlg8RKcHyISIlWD5EpATLh4iUYPkQkRIsHyJSguVDREqwfIhICZYPESkRUflUVlZi5syZ0DQNmqbB6XRi9+7d+vbe3l64XC6MGTMGqampKC0tRVtbW8j3aG5uRklJCZKTk5GVlYV169ZhYGBgeO4NEcWNiMpnwoQJ2LhxI+rq6nD06FH86Ec/woMPPojGxkYAwJo1a/D2229j+/btqK2tRWtrKx566CH99oFAACUlJejr68OBAwewbds2bN26FRs2bBjee0VEsU+GKCMjQ1555RXp7OwUs9ks27dv17edOnVKAIjb7RYRkV27donRaBSPx6PPVFZWiqZp4vf7v/Fn9Pb2itfr1ZeWlhYBIF6vd6jxiWgYeb3esB+bt3zMJxAIoKqqCj09PXA6nairq0N/fz+Kior0malTpyI3NxdutxsA4Ha7MWPGDNjtdn2muLgYPp9Pf/Z0IxUVFbDZbPqSk5Nzq7GJKEZEXD4NDQ1ITU2F1WrFE088gR07diA/Px8ejwcWiwXp6ekh83a7HR6PBwDg8XhCimdw++C2b1JeXg6v16svLS0tkcYmohgT8TWc77zzTtTX18Pr9eKNN95AWVkZamtro5FNZ7VaYeXH5hKNKhGXj8ViweTJkwEABQUFOHLkCF588UU8/PDD6OvrQ2dnZ8izn7a2NjgcDgCAw+HA4cOHQ77f4NmwwRki+m4Y8ut8gsEg/H4/CgoKYDabUVNTo29rampCc3MznE4nAMDpdKKhoQHt7e36THV1NTRNQ35+/lCjEFEcieiZT3l5Oe6//37k5uaiq6sLr732Gvbt24f33nsPNpsNy5Ytw9q1a5GZmQlN07By5Uo4nU7MmzcPALBo0SLk5+fjkUcewaZNm+DxePD000/D5XJxt4roOyai8mlvb8evfvUrnDt3DjabDTNnzsR7772HH//4xwCAF154AUajEaWlpfD7/SguLsaWLVv02yckJGDnzp1Yvnw5nE4nUlJSUFZWhmeffXZ47xURxTyDyJXPqo0jPp8PNpsNXq8XmqapjkNEV0Ty2OR7u4hICZYPESnB8iEiJVg+RKQEy4eIlGD5EJESLB8iUoLlQ0RKsHyISAmWDxEpwfIhIiVYPkSkBMuHiJRg+RCREiwfIlKC5UNESrB8iEgJlg8RKcHyISIlWD5EpATLh4iUYPkQkRIsHyJSguVDREqwfIhICZYPESnB8iEiJVg+RKQEy4eIlGD5EJESLB8iUoLlQ0RKsHyISAmWDxEpwfIhIiWGVD4bN26EwWDA6tWr9XW9vb1wuVwYM2YMUlNTUVpaira2tpDbNTc3o6SkBMnJycjKysK6deswMDAwlChEFGduuXyOHDmCv/3tb5g5c2bI+jVr1uDtt9/G9u3bUVtbi9bWVjz00EP69kAggJKSEvT19eHAgQPYtm0btm7dig0bNtz6vSCi+CO3oKurS6ZMmSLV1dWyYMECWbVqlYiIdHZ2itlslu3bt+uzp06dEgDidrtFRGTXrl1iNBrF4/HoM5WVlaJpmvj9/hv+vN7eXvF6vfrS0tIiAMTr9d5KfCKKEq/XG/Zj85ae+bhcLpSUlKCoqChkfV1dHfr7+0PWT506Fbm5uXC73QAAt9uNGTNmwG636zPFxcXw+XxobGy84c+rqKiAzWbTl5ycnFuJTUQxJOLyqaqqwrFjx1BRUXHdNo/HA4vFgvT09JD1drsdHo9Hn7m6eAa3D267kfLycni9Xn1paWmJNDYRxRhTJMMtLS1YtWoVqqurkZiYGK1M17FarbBarSP284go+iJ65lNXV4f29nbMnj0bJpMJJpMJtbW1eOmll2AymWC329HX14fOzs6Q27W1tcHhcAAAHA7HdWe/Br8enCGi0S+i8lm4cCEaGhpQX1+vL3PmzMHSpUv1P5vNZtTU1Oi3aWpqQnNzM5xOJwDA6XSioaEB7e3t+kx1dTU0TUN+fv4w3S0iinUR7XalpaVh+vTpIetSUlIwZswYff2yZcuwdu1aZGZmQtM0rFy5Ek6nE/PmzQMALFq0CPn5+XjkkUewadMmeDwePP3003C5XNy1IvoOiah8wvHCCy/AaDSitLQUfr8fxcXF2LJli749ISEBO3fuxPLly+F0OpGSkoKysjI8++yzwx2FiGKYQUREdYhI+Xw+2Gw2eL1eaJqmOg4RXRHJY5Pv7SIiJVg+RKQEy4eIlGD5EJESLB8iUoLlQ0RKsHyISAmWDxEpwfIhIiVYPkSkBMuHiJRg+RCREiwfIlKC5UNESrB8iEgJlg8RKcHyISIlWD5EpATLh4iUYPkQkRIsHyJSguVDREqwfIhICZYPESnB8iEiJVg+RKQEy4eIlGD5EJESLB8iUoLlQ0RKsHyISAmWDxEpwfIhIiVYPkSkBMuHiJRg+RCREiwfIlLCpDrArRARAIDP51OchIiuNviYHHyM3kxcls+FCxcAADk5OYqTENGNdHV1wWaz3XQmLssnMzMTANDc3PytdzDW+Hw+5OTkoKWlBZqmqY4TNuYeWfGaW0TQ1dWF7Ozsb52Ny/IxGr8+VGWz2eLqF3M1TdPiMjtzj6x4zB3uEwIecCYiJVg+RKREXJaP1WrFM888A6vVqjpKxOI1O3OPrHjNHQmDhHNOjIhomMXlMx8iin8sHyJSguVDREqwfIhICZYPESkRl+WzefNmTJo0CYmJiSgsLMThw4eV5tm/fz8eeOABZGdnw2Aw4M033wzZLiLYsGEDxo8fj6SkJBQVFeH06dMhMx0dHVi6dCk0TUN6ejqWLVuG7u7uqOauqKjA3LlzkZaWhqysLCxevBhNTU0hM729vXC5XBgzZgxSU1NRWlqKtra2kJnm5maUlJQgOTkZWVlZWLduHQYGBqKWu7KyEjNnztRf/et0OrF79+6YznwjGzduhMFgwOrVq+Mu+7CQOFNVVSUWi0X+/ve/S2Njozz22GOSnp4ubW1tyjLt2rVL/vCHP8g///lPASA7duwI2b5x40ax2Wzy5ptvykcffSQ/+9nPJC8vTy5fvqzP3HfffTJr1iw5ePCgfPDBBzJ58mRZsmRJVHMXFxfLq6++KidOnJD6+nr5yU9+Irm5udLd3a3PPPHEE5KTkyM1NTVy9OhRmTdvntxzzz369oGBAZk+fboUFRXJ8ePHZdeuXTJ27FgpLy+PWu5//etf8s4778inn34qTU1N8tRTT4nZbJYTJ07EbOZrHT58WCZNmiQzZ86UVatW6evjIftwibvyufvuu8XlculfBwIByc7OloqKCoWp/te15RMMBsXhcMhzzz2nr+vs7BSr1Sqvv/66iIicPHlSAMiRI0f0md27d4vBYJCzZ8+OWPb29nYBILW1tXpOs9ks27dv12dOnTolAMTtdovI18VrNBrF4/HoM5WVlaJpmvj9/hHLnpGRIa+88kpcZO7q6pIpU6ZIdXW1LFiwQC+feMg+nOJqt6uvrw91dXUoKirS1xmNRhQVFcHtditM9s3OnDkDj8cTktlms6GwsFDP7Ha7kZ6ejjlz5ugzRUVFMBqNOHTo0Ihl9Xq9AP73qgF1dXXo7+8PyT516lTk5uaGZJ8xYwbsdrs+U1xcDJ/Ph8bGxqhnDgQCqKqqQk9PD5xOZ1xkdrlcKCkpCckIxMff93CKq3e1f/XVVwgEAiF/8QBgt9vxySefKEp1cx6PBwBumHlwm8fjQVZWVsh2k8mEzMxMfSbagsEgVq9ejfnz52P69Ol6LovFgvT09Jtmv9F9G9wWLQ0NDXA6nejt7UVqaip27NiB/Px81NfXx2xmAKiqqsKxY8dw5MiR67bF8t93NMRV+VD0uFwunDhxAh9++KHqKGG58847UV9fD6/XizfeeANlZWWora1VHeumWlpasGrVKlRXVyMxMVF1HOXiardr7NixSEhIuO7of1tbGxwOh6JUNzeY62aZHQ4H2tvbQ7YPDAygo6NjRO7XihUrsHPnTuzduxcTJkzQ1zscDvT19aGzs/Om2W903wa3RYvFYsHkyZNRUFCAiooKzJo1Cy+++GJMZ66rq0N7eztmz54Nk8kEk8mE2tpavPTSSzCZTLDb7TGbPRriqnwsFgsKCgpQU1OjrwsGg6ipqYHT6VSY7Jvl5eXB4XCEZPb5fDh06JCe2el0orOzE3V1dfrMnj17EAwGUVhYGLVsIoIVK1Zgx44d2LNnD/Ly8kK2FxQUwGw2h2RvampCc3NzSPaGhoaQ8qyuroamacjPz49a9msFg0H4/f6Yzrxw4UI0NDSgvr5eX+bMmYOlS5fqf47V7FGh+oh3pKqqqsRqtcrWrVvl5MmT8vjjj0t6enrI0f+R1tXVJcePH5fjx48LAHn++efl+PHj8uWXX4rI16fa09PT5a233pKPP/5YHnzwwRuear/rrrvk0KFD8uGHH8qUKVOifqp9+fLlYrPZZN++fXLu3Dl9uXTpkj7zxBNPSG5uruzZs0eOHj0qTqdTnE6nvn3w1O+iRYukvr5e3n33XRk3blxUT/0++eSTUltbK2fOnJGPP/5YnnzySTEYDPL+++/HbOZvcvXZrnjLPlRxVz4iIn/5y18kNzdXLBaL3H333XLw4EGlefbu3SsArlvKyspE5OvT7evXrxe73S5Wq1UWLlwoTU1NId/jwoULsmTJEklNTRVN0+TRRx+Vrq6uqOa+UWYA8uqrr+ozly9flt/+9reSkZEhycnJ8vOf/1zOnTsX8n2++OILuf/++yUpKUnGjh0rv/vd76S/vz9quX/zm9/IxIkTxWKxyLhx42ThwoV68cRq5m9ybfnEU/ah4vV8iEiJuDrmQ0SjB8uHiJRg+RCREiwfIlKC5UNESrB8iEgJlg8RKcHyISIlWD5EpATLh4iUYPkQkRL/H5WltCPgD9WDAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('Pendulum-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, terminated, truncated, info = self.env.step(\n",
    "            [action * 2])\n",
    "        over = terminated or truncated\n",
    "\n",
    "        #偏移reward,便于训练\n",
    "        reward = (reward + 8) / 8\n",
    "\n",
    "        #限制最大步数\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            over = True\n",
    "\n",
    "        return state, reward, over\n",
    "\n",
    "    #打印游戏图像\n",
    "    def show(self):\n",
    "        from matplotlib import pyplot as plt\n",
    "        plt.figure(figsize=(3, 3))\n",
    "        plt.imshow(self.env.render())\n",
    "        plt.show()\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()\n",
    "\n",
    "env.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e64c315e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[ 0.0763,  0.0989, -0.1105],\n",
       "         [ 0.0565,  0.0827, -0.1343],\n",
       "         [ 0.0818,  0.1618, -0.1522],\n",
       "         [ 0.1293,  0.1526, -0.1936],\n",
       "         [ 0.1190,  0.1070, -0.1644]], grad_fn=<AddBackward0>),\n",
       " tensor([[-0.0517],\n",
       "         [-0.0228],\n",
       "         [-0.0824],\n",
       "         [ 0.0954],\n",
       "         [ 0.0202]], grad_fn=<TanhBackward0>))"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "\n",
    "class FakeEnv(torch.nn.Module):\n",
    "\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.s = torch.nn.Sequential(\n",
    "            torch.nn.Linear(4, 64),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(64, 64),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(64, 64),\n",
    "            torch.nn.ReLU(),\n",
    "        )\n",
    "\n",
    "        self.next_state = torch.nn.Linear(64, 3)\n",
    "        self.reward = torch.nn.Sequential(\n",
    "            torch.nn.Linear(64, 1),\n",
    "            torch.nn.Tanh(),\n",
    "        )\n",
    "\n",
    "    def forward(self, state, action):\n",
    "        state = self.s(torch.cat([state, action], dim=1))\n",
    "        return self.next_state(state) + state[:, :3], self.reward(state)\n",
    "\n",
    "\n",
    "fake_env = FakeEnv()\n",
    "\n",
    "fake_env(torch.randn(5, 3), torch.randn(5, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fd93d2f2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8862467149389309"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "\n",
    "#环境学习阶段使用随机动作即可\n",
    "def get_action(state):\n",
    "    return random.normalvariate(mu=0, sigma=1)\n",
    "\n",
    "\n",
    "get_action(None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "eeeab6f0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-17.940597483586142"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython import display\n",
    "\n",
    "\n",
    "#玩一局游戏并记录数据\n",
    "def play(show=False):\n",
    "    data = []\n",
    "    reward_sum = 0\n",
    "\n",
    "    state = env.reset()\n",
    "    over = False\n",
    "    while not over:\n",
    "        action = get_action(state)\n",
    "\n",
    "        next_state, reward, over = env.step(action)\n",
    "\n",
    "        data.append((state, action, reward, next_state, over))\n",
    "        reward_sum += reward\n",
    "\n",
    "        state = next_state\n",
    "\n",
    "        if show:\n",
    "            display.clear_output(wait=True)\n",
    "            env.show()\n",
    "\n",
    "    return data, reward_sum\n",
    "\n",
    "\n",
    "play()[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9c383bd4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_30896/741254263.py:27: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at  ../torch/csrc/utils/tensor_new.cpp:201.)\n",
      "  state = torch.FloatTensor([i[0] for i in data]).reshape(-1, 3)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(200,\n",
       " (array([-0.9725334 ,  0.23276345,  0.6766915 ], dtype=float32),\n",
       "  -1.0031992633485094,\n",
       "  -0.06231842345025762,\n",
       "  array([-0.97857887,  0.2058723 ,  0.55126405], dtype=float32),\n",
       "  False))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#数据池\n",
    "class Pool:\n",
    "\n",
    "    def __init__(self):\n",
    "        self.pool = []\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.pool)\n",
    "\n",
    "    def __getitem__(self, i):\n",
    "        return self.pool[i]\n",
    "\n",
    "    #更新动作池\n",
    "    def update(self):\n",
    "        #每次更新不少于N条新数据\n",
    "        old_len = len(self.pool)\n",
    "        while len(pool) - old_len < 200:\n",
    "            self.pool.extend(play()[0])\n",
    "\n",
    "        #只保留最新的N条数据\n",
    "        self.pool = self.pool[-2_0000:]\n",
    "\n",
    "    #获取一批数据样本\n",
    "    def sample(self):\n",
    "        data = random.sample(self.pool, 64)\n",
    "\n",
    "        state = torch.FloatTensor([i[0] for i in data]).reshape(-1, 3)\n",
    "        action = torch.FloatTensor([i[1] for i in data]).reshape(-1, 1)\n",
    "        reward = torch.FloatTensor([i[2] for i in data]).reshape(-1, 1)\n",
    "        next_state = torch.FloatTensor([i[3] for i in data]).reshape(-1, 3)\n",
    "        over = torch.LongTensor([i[4] for i in data]).reshape(-1, 1)\n",
    "\n",
    "        return state, action, reward, next_state, over\n",
    "\n",
    "\n",
    "pool = Pool()\n",
    "pool.update()\n",
    "pool.sample()\n",
    "\n",
    "len(pool), pool[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b748ed1d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 400 0.002726170001551509 0.0034544081427156925 10.500285682848135\n",
      "20 4400 0.0005868636653758585 0.00016088939446490258 75.48046000174062\n",
      "40 8400 0.00025014541461132467 0.00010767048661364242 40.06568712378257\n",
      "60 12400 0.0006259015644900501 0.00010308342461939901 65.8760412209607\n",
      "80 16400 9.059047442860901e-05 5.647194120683707e-05 92.80258071329621\n",
      "100 20000 0.00010283117444487289 8.087384048849344e-05 31.64233141467606\n",
      "120 20000 0.0002828323340509087 6.70949521008879e-05 45.58520385601917\n",
      "140 20000 0.00020551285706460476 5.288568354444578e-05 82.84478483553508\n",
      "160 20000 0.0002938929828815162 0.00010266354365739971 65.76003958624305\n",
      "180 20000 7.851848931750283e-05 4.320243533584289e-05 56.1610114488039\n"
     ]
    }
   ],
   "source": [
    "#训练\n",
    "def train(epochs, test_epoch):\n",
    "    fake_env.train()\n",
    "    optimizer = torch.optim.Adam(fake_env.parameters(), lr=1e-3)\n",
    "    loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "    #共更新N轮数据\n",
    "    for epoch in range(epochs):\n",
    "        pool.update()\n",
    "\n",
    "        #每次更新数据后,训练N次\n",
    "        for i in range(200):\n",
    "\n",
    "            #采样N条数据\n",
    "            state, action, reward, next_state, over = pool.sample()\n",
    "\n",
    "            #模型计算\n",
    "            p_next_state, p_reward = fake_env(state, action)\n",
    "\n",
    "            loss_next_state = loss_fn(p_next_state, next_state)\n",
    "            loss_reward = loss_fn(p_reward, reward)\n",
    "            (loss_next_state + loss_reward).backward()\n",
    "            optimizer.step()\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "        if epoch % test_epoch == 0:\n",
    "            print(epoch, len(pool), loss_next_state.item(), loss_reward.item(),\n",
    "                  play()[-1])\n",
    "\n",
    "\n",
    "#环境学习阶段\n",
    "train(200, 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cf277041",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1.2537051439285278"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#使用虚拟环境获取最优动作\n",
    "def get_action(state):\n",
    "    #初始化N步动作的分布\n",
    "    mu = torch.zeros(1, 15)\n",
    "    sigma = torch.ones(1, 15)\n",
    "\n",
    "    state = torch.FloatTensor(state).reshape(1, -1).repeat(50, 1)\n",
    "    state_clone = state.clone()\n",
    "\n",
    "    #反复优化N次动作的分布\n",
    "    for _ in range(5):\n",
    "        #根据N步动作的分布抽样生成N份动作链\n",
    "        action = mu + torch.randn(50, 15) * sigma\n",
    "        reward_sum = torch.zeros(50, 1)\n",
    "        state = state_clone\n",
    "\n",
    "        #按顺序执行N步的动作,计算Q\n",
    "        for i in range(15):\n",
    "            state, reward = fake_env(state, action[:, i].unsqueeze(dim=1))\n",
    "            reward_sum += reward * 0.95**i\n",
    "\n",
    "        #求分数最高的N份动作链\n",
    "        sort = reward_sum.flatten().sort(descending=True).indices\n",
    "        action = action[sort][:10]\n",
    "\n",
    "        #修正动作链的分布\n",
    "        mu = 0.5 * mu + 0.5 * action.mean(dim=0, keepdim=True)\n",
    "        sigma = 0.5 * sigma + 0.5 * action.std(dim=0, keepdim=True)\n",
    "\n",
    "    #返回最优动作\n",
    "    return mu[0, 0].item()\n",
    "\n",
    "\n",
    "get_action(torch.randn(1, 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "86153c62",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 20000 4.735264883493073e-05 6.272528116824105e-05 184.5788197796783\n",
      "1 20000 0.0001188941314467229 4.5524706365540624e-05 183.58295882971538\n",
      "2 20000 0.00011217351857339963 2.686530933715403e-05 168.53254854354407\n",
      "3 20000 5.298547330312431e-05 4.754511974169873e-05 184.28078113247653\n",
      "4 20000 7.20681346138008e-05 2.5564015231793746e-05 152.4564468631718\n",
      "5 20000 0.0002979194978252053 0.00015095982234925032 168.47206662447832\n",
      "6 20000 0.00015631772112101316 0.0002335681492695585 183.68430699254807\n",
      "7 20000 0.00020789820700883865 3.983889109804295e-05 183.3188395321095\n",
      "8 20000 7.334992551477626e-05 4.645865556085482e-05 159.81474878790993\n",
      "9 20000 4.5181670429883525e-05 2.8182716050650924e-05 169.60035595148403\n"
     ]
    }
   ],
   "source": [
    "#动作学习阶段\n",
    "train(10, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9a9e2ad9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAR8AAAEXCAYAAACUBEAgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAd70lEQVR4nO3dfXBU9d028Otskt0kJGeTALtrJAE6oJACUQOErffUPpISbWqlxtYyjE0pg4904QbpcNe0gk9tp6E4UystL52xFb2nmE6saKGApkFDHZa3QDS8Ra1oUmE34SW7CZDNy36fPzSnrATMhmx+u8n1mTkz5Px+u7kOspdnz9lzVhMRARHRIDOpDkBEwxPLh4iUYPkQkRIsHyJSguVDREqwfIhICZYPESnB8iEiJVg+RKQEy4eIlFBWPuvXr8e4ceOQmJiI/Px8HDhwQFUUIlJASfn85S9/wYoVK/Dkk0/i8OHDyM3NRWFhIZqamlTEISIFNBUXlubn52PGjBn4/e9/DwAIBoPIysrC0qVL8fjjj3/h44PBIE6fPo3U1FRomhbpuETURyKC1tZWZGZmwmS6/r5N/CBlMnR0dKCmpgalpaXGOpPJhIKCArjd7l4fEwgEEAgEjJ8/+eQT5OTkRDwrEfVPY2MjxowZc905g14+Z8+eRXd3N+x2e8h6u92OkydP9vqYsrIy/PznP79qfWNjI3Rdj0hOIgqf3+9HVlYWUlNTv3DuoJdPf5SWlmLFihXGzz0bqOs6y4coCvXlcMigl8+oUaMQFxcHr9cbst7r9cLhcPT6GIvFAovFMhjxiGiQDPrZLrPZjLy8PFRVVRnrgsEgqqqq4HQ6BzsOESmi5G3XihUrUFJSgunTp2PmzJn47W9/i4sXL2LBggUq4hCRAkrK56GHHkJzczNWr14Nj8eD2267Dbt27brqIDQRDV1KPudzo/x+P6xWK3w+Hw84E0WRcF6bvLaLiJRg+RCREiwfIlKC5UNESrB8iEgJlg8RKcHyISIlWD5EpATLh4iUYPkQkRIsHyJSguVDREqwfIhICZYPESnB8iEiJVg+RKQEy4eIlGD5EJESLB8iUoLlQ0RKsHyISAmWDxEpwfIhIiVYPkSkBMuHiJRg+RCREiwfIlKC5UNESrB8iEgJlg8RKcHyISIlWD5EpATLh4iUYPkQkRIsHyJSIuzy2bNnD+677z5kZmZC0zS8+uqrIeMigtWrV+Omm25CUlISCgoK8P7774fMOX/+PObPnw9d15GWloaFCxeira3thjaEiGJL2OVz8eJF5ObmYv369b2Or127FuvWrcOmTZuwf/9+jBgxAoWFhWhvbzfmzJ8/H8eOHUNlZSW2b9+OPXv24JFHHun/VhBR7JEbAEC2bt1q/BwMBsXhcMjTTz9trGtpaRGLxSIvvfSSiIgcP35cAMjBgweNOTt37hRN0+STTz7p0+/1+XwCQHw+343EJ6IBFs5rc0CP+Zw6dQoejwcFBQXGOqvVivz8fLjdbgCA2+1GWloapk+fbswpKCiAyWTC/v37e33eQCAAv98fshBRbBvQ8vF4PAAAu90est5utxtjHo8HNpstZDw+Ph4ZGRnGnM8rKyuD1Wo1lqysrIGMTUQKxMTZrtLSUvh8PmNpbGxUHYmIbtCAlo/D4QAAeL3ekPVer9cYczgcaGpqChnv6urC+fPnjTmfZ7FYoOt6yEJEsW1Ay2f8+PFwOByoqqoy1vn9fuzfvx9OpxMA4HQ60dLSgpqaGmPO7t27EQwGkZ+fP5BxiCiKxYf7gLa2NnzwwQfGz6dOnUJtbS0yMjKQnZ2N5cuX45e//CUmTpyI8ePHY9WqVcjMzMTcuXMBAJMnT8Y999yDRYsWYdOmTejs7MSSJUvwve99D5mZmQO2YUQU5cI9lfbmm28KgKuWkpISEfn0dPuqVavEbreLxWKR2bNnS319fchznDt3TubNmycpKSmi67osWLBAWltb+5yBp9qJolM4r01NRERh9/WL3++H1WqFz+fj8R+iKBLOazMmznYR0dDD8iEiJVg+RKRE2Ge7iPpDRAARdDQ3o+PcOUAE5pEjYR49GjCZoGma6og0yFg+FHEigs4LF+DduhUte/ei4/x5QAQJ6elImzkTjgcfRMKoUSygYYblQxHVUzwfr1sH/5EjwBUnVzvPnUPzzp24/O9/Y9yyZTCPHs0CGkZ4zIciKxjEmZdeuqp4rtR29ChO/+//Qrq7BzkcqcTyoYi6/PHHuPD229csHgCACFoOHMClz93xkoY2lg9FVOeFC+i+ePEL5wUvX0bzrl2QYHAQUlE0YPlQ1Lh86hSks1N1DBokLB+KGsHOTgQ7OlTHoEHC8qGoISyfYYXlQ1Gjq60NXS0tqmPQIGH5UETFp6UhbsSIPs0NXrqErtbWCCeiaMHyoYhKSE9HXEqK6hgUhVg+FFEmsxlaPD9IT1dj+VBEmSwWmBIS+jxfuroQg/e3o35g+VBEaXFxgKnv/8y6L12KYBqKJiwfiipdffg0NA0NLB+KKjzVPnywfCji4hIT+zz34smTEUxC0YTlQ5GlaRgxeXKfp/Ng8/DB8qGI6+uHDGl4YflQxMWHUz7c8xk2WD4UcaYwjvlIVxfvaDhMsHwoojRNC+tzPsFAgPf0GSZYPhRVgh0dLJ9hguVDERfO91F0NDej88KFiGWh6MHyoYiz3HQT4pKT+zQ32N7OG4oNEywfirh4XYdmNquOQVGG5UMRZ7JYeFsNugrLhyLOlJj46dXtfcRT7cMDy4ciTouLC+trkPvyPV8U+1g+FF1EeE+fYYLlQ1Gnu61NdQQaBGGVT1lZGWbMmIHU1FTYbDbMnTsX9fX1IXPa29vhcrkwcuRIpKSkoLi4GF6vN2ROQ0MDioqKkJycDJvNhpUrV6Krq+vGt4aik6YhPi2tz9Mv8jvbh4Wwyqe6uhoulwv79u1DZWUlOjs7MWfOHFy84j36Y489hm3btqGiogLV1dU4ffo0HnjgAWO8u7sbRUVF6OjowN69e/HCCy9g8+bNWL169cBtFUUVzWRC0tixfZ4fbG+PYBqKFprcwA1UmpubYbPZUF1dja9+9avw+XwYPXo0tmzZggcffBAAcPLkSUyePBlutxuzZs3Czp078c1vfhOnT5+G3W4HAGzatAk/+clP0NzcDHMfPg/i9/thtVrh8/mg63p/49MgkWAQn7z4IryvvNKn+en/9V/40v/8T4RTUSSE89q8oWM+Pp8PAJCRkQEAqKmpQWdnJwoKCow5kyZNQnZ2NtxuNwDA7XZj6tSpRvEAQGFhIfx+P44dO9br7wkEAvD7/SELxRBN6/MnnHvwpmJDX7/LJxgMYvny5bjzzjsxZcoUAIDH44HZbEba597f2+12eDweY86VxdMz3jPWm7KyMlitVmPJysrqb2xSJJzbagQ7OoBgMIJpKBr0u3xcLheOHj2K8vLygczTq9LSUvh8PmNpbGyM+O+kgRPOZ3yAz26rwRMQQ16/PvO+ZMkSbN++HXv27MGYMWOM9Q6HAx0dHWhpaQnZ+/F6vXA4HMacAwcOhDxfz9mwnjmfZ7FYYLFY+hOVYlAwEOCnnIeBsPZ8RARLlizB1q1bsXv3bowfPz5kPC8vDwkJCaiqqjLW1dfXo6GhAU6nEwDgdDpRV1eHpqYmY05lZSV0XUdOTs6NbAtFsXAur+g8dw7dly9HMA1Fg7D2fFwuF7Zs2YLXXnsNqampxjEaq9WKpKQkWK1WLFy4ECtWrEBGRgZ0XcfSpUvhdDoxa9YsAMCcOXOQk5ODhx9+GGvXroXH48ETTzwBl8vFvZshLGnsWJgSE/t0Gr3zwgWebh8GwiqfjRs3AgC+9rWvhax//vnn8YMf/AAA8Mwzz8BkMqG4uBiBQACFhYXYsGGDMTcuLg7bt2/H4sWL4XQ6MWLECJSUlOCpp566sS2hqBaXkhLW3g8NfWGVT19OfyYmJmL9+vVYv379NeeMHTsWO3bsCOdXU4yLS0oK617ONPTxXwMNClNiIrQwyocHnIc+lg8NCi0uDgjnthq8sn3IY/lQ1BHeVmNYYPlQ9BFB12eX7tDQxfKhQaHFx8Nis/VtcjDI22oMA7yrNw0KLS4OZpsNF99776qx5vZ2/PWjj5BgMmFcSgqmpKdjVHc3RCTsSzModnDPhwaFZjJd88r2dLMZc8eOxVcdDnSJYMuHH+JPbje8Xi+vbh/CWD40ODQNpqSkXofiTSY4kpIwUdcxJzMTP5o0CeNMJvzqV7/CO++8wwIaolg+NDg0DaY+XD6jaRoscXH42u23Y9l//zdefPFFHD16lAU0BLF8aFCEfVuN9naMHzsWixcvxh//+EecP38+QslIFZYPRaXg5ctAMIgJEybg7rvvRkVFBfd+hhiWD0XUlYURl5jY5085t//73+hub4emaZg9ezbee+89nD17NlIxSQGWD0WMiKC5uRndn12nlfSlL4V8Z7uI4EIggENnz+J9vx/BK4qqq63NuL4rOTkZ06ZNw5EjRwZ3AyiiWD4UMd3d3fj1r3+NhoYGAJ9d2f7Zno+IoOHiRSw7cACuffvwf/fuRfmpU+ju5a2VpmnIzc295hcMUGxi+VDENDQ04K9//SveeOMNiAjikpONK9sFwK/r6nC8pQXdIvB3duL3J07g6IULvT6XzWZDS0vL4IWniGP5UMS8++67SEpKQm1tLS5fvoy4ESNCTrf7OztD5ncEgwhc41YaFouFB5yHGJYPRUx+fj6++93vYsGCBYiLi0NCRgYy7roLAKAB+D8OB+KvOAB9i65jbEoKACDOYgm5/087b6s65PDaLooYh8MBm80GETHuzz2qoADndu9Gd1sbSiZMQGpCAv5x5gxuSkrColtuge2z7/dKv+suxF/xjZfNzc1IT09Xsh0UGdzzoYjKzc3F4cOHjbdMidnZuGnePGhmM+JNJnxn3Dhscjrx/267DTcnJ0PTNKTk5MA+d+5/jg+J4J133jG+nJKGBpYPRYymaZg6dSo++OAD46u1NZMJo++5BzeXlCAhIwOayQSTpkHTNJjMZlhnzsTYZctg/uwruAHg0qVLqKurw+23365qUygC+LaLIkrXdeTm5uL111/Hd77zHZhMJpgSEmArKoL1jjvgP3IEAY8HpqQkpEyejJTJkz89Jf8ZEcE//vEP3HLLLRg5cqTCLaGBxvKhiNI0Dffffz9+8YtfYMqUKcjJyYGmadBMJiTefDMSb775mo8VEbz//vt46623sGrVKt7bZ4jh2y6KOF3XsWjRIjz33HM4efJkn06Ziwj+9a9/YcOGDVi4cCEPNg9BLB+KOE3TcMstt2DRokXYtGkTtm3bhkvXuEG8iKC9vR27du3CunXrsGDBAnz5y1/mXs8QpEkMfnLL7/fDarXC5/NBv+J0LEU3EcG5c+fw5z//GR9++CHy8vIwdepUpKenQ9M0XLhwAceOHUNNTQ3GjBmD+fPnw2azsXhiSDivTZYPDToRwZkzZ3Do0CEcP34cfr8fAJCamopJkyZhxowZyMzMhInfcBpzWD5EpEQ4r03+r4WIlGD5EJESLB8iUoLlQ0RKsHyISAmWDxEpwfIhIiVYPkSkBMuHiJQIq3w2btyIadOmQdd16LoOp9OJnTt3GuPt7e1wuVwYOXIkUlJSUFxcDK/XG/IcDQ0NKCoqQnJyMmw2G1auXImurq6B2Roiihlhlc+YMWOwZs0a1NTU4NChQ7j77rtx//33G9+n9Nhjj2Hbtm2oqKhAdXU1Tp8+jQceeMB4fHd3N4qKitDR0YG9e/fihRdewObNm7F69eqB3Soiin5yg9LT0+W5556TlpYWSUhIkIqKCmPsxIkTAkDcbreIiOzYsUNMJpN4PB5jzsaNG0XXdQkEAtf8He3t7eLz+YylsbFRAIjP57vR+EQ0gHw+X59fm/0+5tPd3Y3y8nJcvHgRTqcTNTU16OzsREFBgTFn0qRJyM7OhtvtBgC43W5MnToVdrvdmFNYWAi/33/db6MsKyuD1Wo1lqysrP7GJqIoEXb51NXVISUlBRaLBY8++ii2bt2KnJwceDwemM1mpKWlhcy32+3weDwAAI/HE1I8PeM9Y9dSWloKn89nLI2NjeHGJqIoE/Y9nG+99VbU1tbC5/Ph5ZdfRklJCaqrqyORzWCxWIzvfSKioSHs8jGbzZgwYQIAIC8vDwcPHsSzzz6Lhx56CB0dHWhpaQnZ+/F6vXA4HAA+/RK5AwcOhDxfz9mwnjlENDzc8Od8gsEgAoEA8vLykJCQgKqqKmOsvr4eDQ0NcDqdAACn04m6ujo0NTUZcyorK6HrOnJycm40ChHFkLD2fEpLS3HvvfciOzsbra2t2LJlC9566y28/vrrsFqtWLhwIVasWIGMjAzouo6lS5fC6XRi1qxZAIA5c+YgJycHDz/8MNauXQuPx4MnnngCLpeLb6uIhpmwyqepqQnf//73cebMGVitVkybNg2vv/46vv71rwMAnnnmGZhMJhQXFyMQCKCwsBAbNmwwHh8XF4ft27dj8eLFcDqdGDFiBEpKSvDUU08N7FYRUdTjPZyJaMDwHs5EFPVYPkSkBMuHiJRg+RCREiwfIlKC5UNESrB8iEgJlg8RKcHyISIlWD5EpATLh4iUYPkQkRIsHyJSguVDREqwfIhICZYPESnB8iEiJVg+RKQEy4eIlGD5EJESLB8iUoLlQ0RKsHyISAmWDxEpwfIhIiVYPkSkBMuHiJRg+RCREiwfIlKC5UNESrB8iEgJlg8RKcHyISIlWD5EpATLh4iUuKHyWbNmDTRNw/Lly4117e3tcLlcGDlyJFJSUlBcXAyv1xvyuIaGBhQVFSE5ORk2mw0rV65EV1fXjUQhohjT7/I5ePAg/vCHP2DatGkh6x977DFs27YNFRUVqK6uxunTp/HAAw8Y493d3SgqKkJHRwf27t2LF154AZs3b8bq1av7vxVEFHukH1pbW2XixIlSWVkpd911lyxbtkxERFpaWiQhIUEqKiqMuSdOnBAA4na7RURkx44dYjKZxOPxGHM2btwouq5LIBDo9fe1t7eLz+czlsbGRgEgPp+vP/GJKEJ8Pl+fX5v92vNxuVwoKipCQUFByPqamhp0dnaGrJ80aRKys7PhdrsBAG63G1OnToXdbjfmFBYWwu/349ixY73+vrKyMlitVmPJysrqT2wiiiJhl095eTkOHz6MsrKyq8Y8Hg/MZjPS0tJC1tvtdng8HmPOlcXTM94z1pvS0lL4fD5jaWxsDDc2EUWZ+HAmNzY2YtmyZaisrERiYmKkMl3FYrHAYrEM2u8josgLa8+npqYGTU1NuOOOOxAfH4/4+HhUV1dj3bp1iI+Ph91uR0dHB1paWkIe5/V64XA4AAAOh+Oqs189P/fMIaKhL6zymT17Nurq6lBbW2ss06dPx/z5840/JyQkoKqqynhMfX09Ghoa4HQ6AQBOpxN1dXVoamoy5lRWVkLXdeTk5AzQZhFRtAvrbVdqaiqmTJkSsm7EiBEYOXKksX7hwoVYsWIFMjIyoOs6li5dCqfTiVmzZgEA5syZg5ycHDz88MNYu3YtPB4PnnjiCbhcLr61IhpGwiqfvnjmmWdgMplQXFyMQCCAwsJCbNiwwRiPi4vD9u3bsXjxYjidTowYMQIlJSV46qmnBjoKEUUxTUREdYhw+f1+WK1W+Hw+6LquOg4RfSac1yav7SIiJVg+RKQEy4eIlGD5EJESLB8iUoLlQ0RKsHyISAmWDxEpwfIhIiVYPkSkBMuHiJRg+RCREiwfIlKC5UNESrB8iEgJlg8RKcHyISIlWD5EpATLh4iUYPkQkRIsHyJSguVDREqwfIhICZYPESnB8iEiJVg+RKQEy4eIlGD5EJESLB8iUoLlQ0RKsHyISAmWDxEpwfIhIiVYPkSkBMuHiJRg+RCREiwfIlIiXnWA/hARAIDf71echIiu1POa7HmNXk9Mls+5c+cAAFlZWYqTEFFvWltbYbVarzsnJssnIyMDANDQ0PCFGxht/H4/srKy0NjYCF3XVcfpM+YeXLGaW0TQ2tqKzMzML5wbk+VjMn16qMpqtcbUf5gr6boek9mZe3DFYu6+7hDwgDMRKcHyISIlYrJ8LBYLnnzySVgsFtVRwhar2Zl7cMVq7nBo0pdzYkREAywm93yIKPaxfIhICZYPESnB8iEiJVg+RKRETJbP+vXrMW7cOCQmJiI/Px8HDhxQmmfPnj247777kJmZCU3T8Oqrr4aMiwhWr16Nm266CUlJSSgoKMD7778fMuf8+fOYP38+dF1HWloaFi5ciLa2tojmLisrw4wZM5CamgqbzYa5c+eivr4+ZE57eztcLhdGjhyJlJQUFBcXw+v1hsxpaGhAUVERkpOTYbPZsHLlSnR1dUUs98aNGzFt2jTj079OpxM7d+6M6sy9WbNmDTRNw/Lly2Mu+4CQGFNeXi5ms1n+9Kc/ybFjx2TRokWSlpYmXq9XWaYdO3bIz372M3nllVcEgGzdujVkfM2aNWK1WuXVV1+Vd955R771rW/J+PHj5fLly8ace+65R3Jzc2Xfvn3yz3/+UyZMmCDz5s2LaO7CwkJ5/vnn5ejRo1JbWyvf+MY3JDs7W9ra2ow5jz76qGRlZUlVVZUcOnRIZs2aJV/5yleM8a6uLpkyZYoUFBTIkSNHZMeOHTJq1CgpLS2NWO6//e1v8ve//13ee+89qa+vl5/+9KeSkJAgR48ejdrMn3fgwAEZN26cTJs2TZYtW2asj4XsAyXmymfmzJnicrmMn7u7uyUzM1PKysoUpvqPz5dPMBgUh8MhTz/9tLGupaVFLBaLvPTSSyIicvz4cQEgBw8eNObs3LlTNE2TTz75ZNCyNzU1CQCprq42ciYkJEhFRYUx58SJEwJA3G63iHxavCaTSTwejzFn48aNouu6BAKBQcuenp4uzz33XExkbm1tlYkTJ0plZaXcddddRvnEQvaBFFNvuzo6OlBTU4OCggJjnclkQkFBAdxut8Jk13bq1Cl4PJ6QzFarFfn5+UZmt9uNtLQ0TJ8+3ZhTUFAAk8mE/fv3D1pWn88H4D93DaipqUFnZ2dI9kmTJiE7Ozsk+9SpU2G32405hYWF8Pv9OHbsWMQzd3d3o7y8HBcvXoTT6YyJzC6XC0VFRSEZgdj4+x5IMXVV+9mzZ9Hd3R3yFw8AdrsdJ0+eVJTq+jweDwD0mrlnzOPxwGazhYzHx8cjIyPDmBNpwWAQy5cvx5133okpU6YYucxmM9LS0q6bvbdt6xmLlLq6OjidTrS3tyMlJQVbt25FTk4OamtrozYzAJSXl+Pw4cM4ePDgVWPR/PcdCTFVPhQ5LpcLR48exdtvv606Sp/ceuutqK2thc/nw8svv4ySkhJUV1erjnVdjY2NWLZsGSorK5GYmKg6jnIx9bZr1KhRiIuLu+rov9frhcPhUJTq+npyXS+zw+FAU1NTyHhXVxfOnz8/KNu1ZMkSbN++HW+++SbGjBljrHc4HOjo6EBLS8t1s/e2bT1jkWI2mzFhwgTk5eWhrKwMubm5ePbZZ6M6c01NDZqamnDHHXcgPj4e8fHxqK6uxrp16xAfHw+73R612SMhpsrHbDYjLy8PVVVVxrpgMIiqqio4nU6Fya5t/PjxcDgcIZn9fj/2799vZHY6nWhpaUFNTY0xZ/fu3QgGg8jPz49YNhHBkiVLsHXrVuzevRvjx48PGc/Ly0NCQkJI9vr6ejQ0NIRkr6urCynPyspK6LqOnJyciGX/vGAwiEAgENWZZ8+ejbq6OtTW1hrL9OnTMX/+fOPP0Zo9IlQf8Q5XeXm5WCwW2bx5sxw/flweeeQRSUtLCzn6P9haW1vlyJEjcuTIEQEgv/nNb+TIkSPy8ccfi8inp9rT0tLktddek3fffVfuv//+Xk+133777bJ//355++23ZeLEiRE/1b548WKxWq3y1ltvyZkzZ4zl0qVLxpxHH31UsrOzZffu3XLo0CFxOp3idDqN8Z5Tv3PmzJHa2lrZtWuXjB49OqKnfh9//HGprq6WU6dOybvvviuPP/64aJomb7zxRtRmvpYrz3bFWvYbFXPlIyLyu9/9TrKzs8VsNsvMmTNl3759SvO8+eabAuCqpaSkREQ+Pd2+atUqsdvtYrFYZPbs2VJfXx/yHOfOnZN58+ZJSkqK6LouCxYskNbW1ojm7i0zAHn++eeNOZcvX5Yf/ehHkp6eLsnJyfLtb39bzpw5E/I8H330kdx7772SlJQko0aNkh//+MfS2dkZsdw//OEPZezYsWI2m2X06NEye/Zso3iiNfO1fL58Yin7jeL9fIhIiZg65kNEQwfLh4iUYPkQkRIsHyJSguVDREqwfIhICZYPESnB8iEiJVg+RKQEy4eIlGD5EJES/x/GnVBPehMq6gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "183.52116816080024"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "play(True)[-1]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:pt39]",
   "language": "python",
   "name": "conda-env-pt39-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
